{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Description:\n",
    "建立Wide&Deep Model在cretio数据集上进行点击率预测的任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.522823Z",
     "start_time": "2020-11-30T03:28:55.793448Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import datetime\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset, TensorDataset\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchkeras import summary, Model\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.538781Z",
     "start_time": "2020-11-30T03:28:57.523820Z"
    }
   },
   "outputs": [],
   "source": [
    "file_path = './preprocessed_data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.554757Z",
     "start_time": "2020-11-30T03:28:57.540789Z"
    }
   },
   "outputs": [],
   "source": [
    "def prepared_data(file_path):\n",
    "    \n",
    "    # 读入训练集，验证集和测试集\n",
    "    train = pd.read_csv(file_path + 'train_set.csv')\n",
    "    val = pd.read_csv(file_path + 'val_set.csv')\n",
    "    test = pd.read_csv(file_path + 'test_set.csv')\n",
    "    \n",
    "    trn_x, trn_y = train.drop(columns='Label').values, train['Label'].values\n",
    "    val_x, val_y = val.drop(columns='Label').values, val['Label'].values\n",
    "    test_x = test.values\n",
    "    \n",
    "    fea_col = np.load(file_path + 'fea_col.npy', allow_pickle=True)\n",
    "    \n",
    "    return fea_col, (trn_x, trn_y), (val_x, val_y), test_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.600615Z",
     "start_time": "2020-11-30T03:28:57.556734Z"
    }
   },
   "outputs": [],
   "source": [
    "fea_cols, (trn_x, trn_y), (val_x, val_y), test_x = prepared_data(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.616575Z",
     "start_time": "2020-11-30T03:28:57.602655Z"
    }
   },
   "outputs": [],
   "source": [
    "# 把数据构建成数据管道\n",
    "dl_train_dataset = TensorDataset(torch.tensor(trn_x).float(), torch.tensor(trn_y).float())\n",
    "dl_val_dataset = TensorDataset(torch.tensor(val_x).float(), torch.tensor(val_y).float())\n",
    "\n",
    "dl_train = DataLoader(dl_train_dataset, shuffle=True, batch_size=32)\n",
    "dl_val = DataLoader(dl_val_dataset, shuffle=True, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.632536Z",
     "start_time": "2020-11-30T03:28:57.618568Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 39]) tensor([0., 0., 1., 0., 1., 1., 0., 0., 1., 1., 0., 1., 0., 0., 0., 0., 1., 0.,\n",
      "        0., 0., 0., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0.])\n"
     ]
    }
   ],
   "source": [
    "# 看一下数据\n",
    "for b in iter(dl_train):\n",
    "    print(b[0].shape, b[1])\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构建模型\n",
    "这里依然是使用继承nn.Module基类构建模型， 并辅助应用模型容器进行封装， Wide&Deep的模型结构如下：\n",
    "\n",
    "![](img/w&d.png)\n",
    "\n",
    "这个模型搭建起来，相对比较简单， 主要分为wide部分， Deep部分和两者的拼接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.648487Z",
     "start_time": "2020-11-30T03:28:57.634525Z"
    }
   },
   "outputs": [],
   "source": [
    "class Linear(nn.Module):\n",
    "    \"\"\"\n",
    "    Linear part\n",
    "    \"\"\"\n",
    "    def __init__(self, input_dim):\n",
    "        super(Linear, self).__init__()\n",
    "        self.linear = nn.Linear(in_features=input_dim, out_features=1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.linear(x)\n",
    "\n",
    "class Dnn(nn.Module):\n",
    "    \"\"\"\n",
    "    Dnn part\n",
    "    \"\"\"\n",
    "    def __init__(self, hidden_units, dropout=0.):\n",
    "        \"\"\"\n",
    "        hidden_units: 列表， 每个元素表示每一层的神经单元个数， 比如[256, 128, 64], 两层网络， 第一层神经单元128， 第二层64， 第一个维度是输入维度\n",
    "        dropout: 失活率\n",
    "        \"\"\"\n",
    "        super(Dnn, self).__init__()\n",
    "        \n",
    "        self.dnn_network = nn.ModuleList([nn.Linear(layer[0], layer[1]) for layer in list(zip(hidden_units[:-1], hidden_units[1:]))])\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        for linear in self.dnn_network:\n",
    "            x = linear(x)\n",
    "            x = F.relu(x)\n",
    "        \n",
    "        x = self.dropout(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.679407Z",
     "start_time": "2020-11-30T03:28:57.652477Z"
    }
   },
   "outputs": [],
   "source": [
    "class WideDeep(nn.Module):\n",
    "    def __init__(self, feature_columns, hidden_units, dnn_dropout=0.):\n",
    "        super(WideDeep, self).__init__()\n",
    "        self.dense_feature_cols, self.sparse_feature_cols = feature_columns\n",
    "        \n",
    "        # embedding \n",
    "        self.embed_layers = nn.ModuleDict({\n",
    "            'embed_' + str(i): nn.Embedding(num_embeddings=feat['feat_num'], embedding_dim=feat['embed_dim'])\n",
    "            for i, feat in enumerate(self.sparse_feature_cols)\n",
    "        })\n",
    "        \n",
    "        hidden_units.insert(0, len(self.dense_feature_cols) + len(self.sparse_feature_cols)*self.sparse_feature_cols[0]['embed_dim'])\n",
    "        self.dnn_network = Dnn(hidden_units)\n",
    "        self.linear = Linear(len(self.dense_feature_cols))\n",
    "        self.final_linear = nn.Linear(hidden_units[-1], 1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        dense_input, sparse_inputs = x[:, :len(self.dense_feature_cols)], x[:, len(self.dense_feature_cols):]\n",
    "        sparse_inputs = sparse_inputs.long()\n",
    "        sparse_embeds = [self.embed_layers['embed_'+str(i)](sparse_inputs[:, i]) for i in range(sparse_inputs.shape[1])]\n",
    "        sparse_embeds = torch.cat(sparse_embeds, axis=-1)\n",
    "        \n",
    "        dnn_input = torch.cat([sparse_embeds, dense_input], axis=-1)\n",
    "        \n",
    "        # Wide\n",
    "        wide_out = self.linear(dense_input)\n",
    "        \n",
    "        # Deep\n",
    "        deep_out = self.dnn_network(dnn_input)\n",
    "        deep_out = self.final_linear(deep_out)\n",
    "        \n",
    "        # out\n",
    "        outputs = F.sigmoid(0.5 * (wide_out + deep_out))\n",
    "        \n",
    "        return outputs  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.710322Z",
     "start_time": "2020-11-30T03:28:57.681400Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "         Embedding-1                    [-1, 8]             632\n",
      "         Embedding-2                    [-1, 8]           2,016\n",
      "         Embedding-3                    [-1, 8]          10,344\n",
      "         Embedding-4                    [-1, 8]           8,344\n",
      "         Embedding-5                    [-1, 8]             240\n",
      "         Embedding-6                    [-1, 8]              56\n",
      "         Embedding-7                    [-1, 8]           9,312\n",
      "         Embedding-8                    [-1, 8]             312\n",
      "         Embedding-9                    [-1, 8]              16\n",
      "        Embedding-10                    [-1, 8]           7,264\n",
      "        Embedding-11                    [-1, 8]           7,408\n",
      "        Embedding-12                    [-1, 8]           9,912\n",
      "        Embedding-13                    [-1, 8]           6,592\n",
      "        Embedding-14                    [-1, 8]             160\n",
      "        Embedding-15                    [-1, 8]           6,552\n",
      "        Embedding-16                    [-1, 8]           9,272\n",
      "        Embedding-17                    [-1, 8]              72\n",
      "        Embedding-18                    [-1, 8]           4,272\n",
      "        Embedding-19                    [-1, 8]           1,608\n",
      "        Embedding-20                    [-1, 8]              32\n",
      "        Embedding-21                    [-1, 8]           9,632\n",
      "        Embedding-22                    [-1, 8]              56\n",
      "        Embedding-23                    [-1, 8]              96\n",
      "        Embedding-24                    [-1, 8]           5,832\n",
      "        Embedding-25                    [-1, 8]             264\n",
      "        Embedding-26                    [-1, 8]           4,432\n",
      "           Linear-27                    [-1, 1]              14\n",
      "           Linear-28                  [-1, 256]          56,832\n",
      "           Linear-29                  [-1, 128]          32,896\n",
      "           Linear-30                   [-1, 64]           8,256\n",
      "          Dropout-31                   [-1, 64]               0\n",
      "           Linear-32                    [-1, 1]              65\n",
      "================================================================\n",
      "Total params: 202,791\n",
      "Trainable params: 202,791\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.000149\n",
      "Forward/backward pass size (MB): 0.005508\n",
      "Params size (MB): 0.773586\n",
      "Estimated Total Size (MB): 0.779243\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# 建立模型\n",
    "hidden_units = [256, 128, 64]\n",
    "dnn_dropout = 0.\n",
    "\n",
    "model = WideDeep(fea_cols, hidden_units, dnn_dropout)\n",
    "summary(model, input_shape=(trn_x.shape[1],))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.726280Z",
     "start_time": "2020-11-30T03:28:57.712318Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.4815],\n",
      "        [0.4876],\n",
      "        [0.4801],\n",
      "        [0.4843],\n",
      "        [0.4831],\n",
      "        [0.4791],\n",
      "        [0.4835],\n",
      "        [0.4839],\n",
      "        [0.4954],\n",
      "        [0.4847],\n",
      "        [0.4830],\n",
      "        [0.4773],\n",
      "        [0.4816],\n",
      "        [0.4828],\n",
      "        [0.4793],\n",
      "        [0.4821],\n",
      "        [0.4852],\n",
      "        [0.4784],\n",
      "        [0.4815],\n",
      "        [0.4805],\n",
      "        [0.4787],\n",
      "        [0.4756],\n",
      "        [0.4787],\n",
      "        [0.4874],\n",
      "        [0.4855],\n",
      "        [0.4827],\n",
      "        [0.4933],\n",
      "        [0.4819],\n",
      "        [0.4777],\n",
      "        [0.4764],\n",
      "        [0.4806],\n",
      "        [0.4697]], grad_fn=<SigmoidBackward>)\n"
     ]
    }
   ],
   "source": [
    "# 测试一下模型\n",
    "for fea, label in iter(dl_train):\n",
    "    out = model(fea)\n",
    "    print(out)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型的训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:28:57.742236Z",
     "start_time": "2020-11-30T03:28:57.728274Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型的相关设置\n",
    "def auc(y_pred, y_true):\n",
    "    pred = y_pred.data\n",
    "    y = y_true.data\n",
    "    return roc_auc_score(y, pred)\n",
    "\n",
    "loss_func = nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)\n",
    "metric_func = auc\n",
    "metric_name = 'auc'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:29:06.013125Z",
     "start_time": "2020-11-30T03:28:57.744232Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_training.........\n",
      "================================================================2020-11-30 11:28:57\n",
      "[step=10] loss: 0.607, auc: 0.489\n",
      "[step=20] loss: 0.594, auc: 0.495\n",
      "[step=30] loss: 0.566, auc: 0.506\n",
      "[step=40] loss: 0.557, auc: 0.529\n",
      "\n",
      "EPOCH=1, loss=0.557, auc = 0.529, val_loss=0.443, val_auc = 0.587\n",
      "\n",
      "================================================================================2020-11-30 11:28:59\n",
      "[step=10] loss: 0.493, auc: 0.796\n",
      "[step=20] loss: 0.489, auc: 0.763\n",
      "[step=30] loss: 0.483, auc: 0.732\n",
      "[step=40] loss: 0.486, auc: 0.727\n",
      "\n",
      "EPOCH=2, loss=0.486, auc = 0.727, val_loss=0.441, val_auc = 0.630\n",
      "\n",
      "================================================================================2020-11-30 11:29:00\n",
      "[step=10] loss: 0.416, auc: 0.811\n",
      "[step=20] loss: 0.426, auc: 0.838\n",
      "[step=30] loss: 0.435, auc: 0.825\n",
      "[step=40] loss: 0.432, auc: 0.825\n",
      "\n",
      "EPOCH=3, loss=0.432, auc = 0.825, val_loss=0.438, val_auc = 0.657\n",
      "\n",
      "================================================================================2020-11-30 11:29:00\n",
      "[step=10] loss: 0.357, auc: 0.933\n",
      "[step=20] loss: 0.341, auc: 0.930\n",
      "[step=30] loss: 0.339, auc: 0.919\n",
      "[step=40] loss: 0.347, auc: 0.912\n",
      "\n",
      "EPOCH=4, loss=0.347, auc = 0.912, val_loss=0.476, val_auc = 0.635\n",
      "\n",
      "================================================================================2020-11-30 11:29:01\n",
      "[step=10] loss: 0.221, auc: 0.986\n",
      "[step=20] loss: 0.206, auc: 0.985\n",
      "[step=30] loss: 0.199, auc: 0.984\n",
      "[step=40] loss: 0.192, auc: 0.984\n",
      "\n",
      "EPOCH=5, loss=0.192, auc = 0.984, val_loss=0.694, val_auc = 0.650\n",
      "\n",
      "================================================================================2020-11-30 11:29:02\n",
      "[step=10] loss: 0.074, auc: 0.999\n",
      "[step=20] loss: 0.068, auc: 0.998\n",
      "[step=30] loss: 0.068, auc: 0.998\n",
      "[step=40] loss: 0.066, auc: 0.998\n",
      "\n",
      "EPOCH=6, loss=0.066, auc = 0.998, val_loss=0.861, val_auc = 0.640\n",
      "\n",
      "================================================================================2020-11-30 11:29:03\n",
      "[step=10] loss: 0.022, auc: 1.000\n",
      "[step=20] loss: 0.022, auc: 1.000\n",
      "[step=30] loss: 0.028, auc: 0.998\n",
      "[step=40] loss: 0.027, auc: 0.998\n",
      "\n",
      "EPOCH=7, loss=0.027, auc = 0.998, val_loss=1.027, val_auc = 0.627\n",
      "\n",
      "================================================================================2020-11-30 11:29:03\n",
      "[step=10] loss: 0.017, auc: 1.000\n",
      "[step=20] loss: 0.011, auc: 1.000\n",
      "[step=30] loss: 0.013, auc: 1.000\n",
      "[step=40] loss: 0.012, auc: 1.000\n",
      "\n",
      "EPOCH=8, loss=0.012, auc = 1.000, val_loss=1.126, val_auc = 0.668\n",
      "\n",
      "================================================================================2020-11-30 11:29:04\n",
      "[step=10] loss: 0.004, auc: 1.000\n",
      "[step=20] loss: 0.006, auc: 1.000\n",
      "[step=30] loss: 0.006, auc: 1.000\n",
      "[step=40] loss: 0.006, auc: 1.000\n",
      "\n",
      "EPOCH=9, loss=0.006, auc = 1.000, val_loss=1.202, val_auc = 0.630\n",
      "\n",
      "================================================================================2020-11-30 11:29:05\n",
      "[step=10] loss: 0.003, auc: 1.000\n",
      "[step=20] loss: 0.013, auc: 1.000\n",
      "[step=30] loss: 0.010, auc: 1.000\n",
      "[step=40] loss: 0.010, auc: 1.000\n",
      "\n",
      "EPOCH=10, loss=0.010, auc = 1.000, val_loss=1.248, val_auc = 0.644\n",
      "\n",
      "================================================================================2020-11-30 11:29:06\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "# 脚本训练风格\n",
    "epochs = 10\n",
    "log_step_freq = 10\n",
    "\n",
    "dfhistory = pd.DataFrame(columns=['epoch', 'loss', metric_name, 'val_loss', 'val_'+metric_name])\n",
    "\n",
    "print('start_training.........')\n",
    "nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "print('========'*8 + '%s' %nowtime)\n",
    "\n",
    "for epoch in range(1, epochs+1):\n",
    "    \n",
    "    # 训练阶段\n",
    "    model.train()\n",
    "    loss_sum = 0.0\n",
    "    metric_sum = 0.0\n",
    "    step = 1\n",
    "    \n",
    "    for step, (features, labels) in enumerate(dl_train, 1):\n",
    "        # 梯度清零\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # 正向传播\n",
    "        predictions = model(features);\n",
    "        loss = loss_func(predictions, labels)\n",
    "        try:\n",
    "            metric = metric_func(predictions, labels)\n",
    "        except ValueError:\n",
    "            pass\n",
    "        \n",
    "        # 反向传播\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # 打印batch级别日志\n",
    "        loss_sum += loss.item()\n",
    "        metric_sum += metric.item()\n",
    "        if step % log_step_freq == 0:\n",
    "            print((\"[step=%d] loss: %.3f, \" + metric_name + \": %.3f\") % (step, loss_sum/step, metric_sum/step));\n",
    "    \n",
    "    # 验证阶段\n",
    "    model.eval()\n",
    "    val_loss_sum = 0.0\n",
    "    val_metric_sum = 0.0\n",
    "    val_step = 1\n",
    "    \n",
    "    for val_step, (features, labels) in enumerate(dl_val, 1):\n",
    "        with torch.no_grad():\n",
    "            predictions = model(features)\n",
    "            val_loss = loss_func(predictions, labels)\n",
    "            try:\n",
    "                val_metric = metric_func(predictions, labels)\n",
    "            except ValueError:\n",
    "                pass\n",
    "        \n",
    "        val_loss_sum += val_loss.item()\n",
    "        val_metric_sum += val_metric.item()\n",
    "    \n",
    "    # 记录日志\n",
    "    info = (epoch, loss_sum/step, metric_sum/step, val_loss_sum/val_step, val_metric_sum/val_step)\n",
    "    dfhistory.loc[epoch-1] = info\n",
    "    \n",
    "    # 打印日志\n",
    "    print((\"\\nEPOCH=%d, loss=%.3f, \" + metric_name + \" = %.3f, val_loss=%.3f, \" + \"val_\" + metric_name + \" = %.3f\") %info)\n",
    "    nowtime = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    print('\\n' + '=========='* 8 + '%s' %nowtime)\n",
    "    \n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:29:06.045035Z",
     "start_time": "2020-11-30T03:29:06.016115Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>loss</th>\n",
       "      <th>auc</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.556742</td>\n",
       "      <td>0.528857</td>\n",
       "      <td>0.443361</td>\n",
       "      <td>0.586776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.486083</td>\n",
       "      <td>0.726870</td>\n",
       "      <td>0.441146</td>\n",
       "      <td>0.629656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.431980</td>\n",
       "      <td>0.825220</td>\n",
       "      <td>0.438139</td>\n",
       "      <td>0.657086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.346778</td>\n",
       "      <td>0.912290</td>\n",
       "      <td>0.475598</td>\n",
       "      <td>0.635332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.192280</td>\n",
       "      <td>0.984185</td>\n",
       "      <td>0.693825</td>\n",
       "      <td>0.650170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.065749</td>\n",
       "      <td>0.998340</td>\n",
       "      <td>0.861433</td>\n",
       "      <td>0.639916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.026761</td>\n",
       "      <td>0.998252</td>\n",
       "      <td>1.026885</td>\n",
       "      <td>0.627445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.011845</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.125897</td>\n",
       "      <td>0.668218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.006337</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.201750</td>\n",
       "      <td>0.629755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.009839</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.247855</td>\n",
       "      <td>0.643955</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch      loss       auc  val_loss   val_auc\n",
       "0    1.0  0.556742  0.528857  0.443361  0.586776\n",
       "1    2.0  0.486083  0.726870  0.441146  0.629656\n",
       "2    3.0  0.431980  0.825220  0.438139  0.657086\n",
       "3    4.0  0.346778  0.912290  0.475598  0.635332\n",
       "4    5.0  0.192280  0.984185  0.693825  0.650170\n",
       "5    6.0  0.065749  0.998340  0.861433  0.639916\n",
       "6    7.0  0.026761  0.998252  1.026885  0.627445\n",
       "7    8.0  0.011845  1.000000  1.125897  0.668218\n",
       "8    9.0  0.006337  1.000000  1.201750  0.629755\n",
       "9   10.0  0.009839  1.000000  1.247855  0.643955"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfhistory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:29:06.501815Z",
     "start_time": "2020-11-30T03:29:06.048028Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xUZfb48c9JISGASO9JUEEBKS5BsGNdRAV/ioiACspiQ9G1i23dZde2svpVYVGRXYkK4iIorKiAYgMJCiq9LCXUgKAgNcn5/fFMzCRMCjAzd8p5v17zmpl779w5cyH33Kfc5xFVxRhjTPxK8DoAY4wx3rJEYIwxcc4SgTHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoEJKhFJFJHdIpIezG29JCIniEjQ+1mLyAUissbv/TIROasy2x7Bd70qIg8d6efL2e9fRGRssPdrwivJ6wCMt0Rkt9/bNGA/UOB7f5OqZh/O/lS1AKge7G3jgaqeGIz9iMggoL+qdvXb96Bg7NvEJksEcU5VfzsR+644B6nqJ2VtLyJJqpofjtiMMeFhVUOmXL6i/3gReUtEdgH9ReQ0EZkjIjtFZJOIvCAiyb7tk0RERSTT936cb/1/RWSXiHwtIs0Pd1vf+otFZLmI/Cwi/yciX4rIgDLirkyMN4nIShHZISIv+H02UURGiMh2EVkFdCvn+DwsIm+XWvaSiDznez1IRJb4fs8q39V6WfvKFZGuvtdpIvKGL7ZFQMcA37vat99FItLDt7wt8CJwlq/abZvfsX3c7/M3+377dhF5T0QaVebYVERELvfFs1NEZorIiX7rHhKRjSLyi4gs9futXUTkW9/yLSLyTGW/zwSJqtrDHqgqwBrgglLL/gIcAC7DXThUBToBnXElyuOA5cAQ3/ZJgAKZvvfjgG1AFpAMjAfGHcG29YFdQE/fuj8CB4EBZfyWysQ4GagJZAI/Ff12YAiwCGgK1AFmuz+VgN9zHLAbqOa3761Alu/9Zb5tBDgP2Au08627AFjjt69coKvv9bPAp0AtIANYXGrb3kAj379JX18MDXzrBgGflopzHPC47/VFvhg7AKnAy8DMyhybAL//L8BY3+tWvjjO8/0bPeQ77slAG2At0NC3bXPgON/recA1vtc1gM5e/y3E28NKBKYyvlDV91W1UFX3quo8VZ2rqvmquhoYDZxTzucnqmqOqh4EsnEnoMPd9lJggapO9q0bgUsaAVUyxr+p6s+qugZ30i36rt7ACFXNVdXtwJPlfM9q4EdcggK4ENipqjm+9e+r6mp1ZgIzgIANwqX0Bv6iqjtUdS3uKt//eyeo6ibfv8mbuCSeVYn9AvQDXlXVBaq6D3gAOEdEmvptU9axKU8fYIqqzvT9Gz0JHINLyPm4pNPGV734P9+xA5fQW4hIHVXdpapzK/k7TJBYIjCVsd7/jYicJCJTRWSziPwCPAHULefzm/1e76H8BuKytm3sH4eqKu4KOqBKxlip78JdyZbnTeAa3+u+uARWFMelIjJXRH4SkZ24q/HyjlWRRuXFICIDRGShrwpmJ3BSJfcL7vf9tj9V/QXYATTx2+Zw/s3K2m8h7t+oiaouA+7G/Tts9VU1NvRtOhBoDSwTkW9EpHslf4cJEksEpjJKd538J+4q+ARVPQZ4FFf1EUqbcFU1AIiIUPLEVdrRxLgJaOb3vqLureOBC3xX1D1xiQERqQpMBP6Gq7Y5FvioknFsLisGETkOGAncAtTx7Xep334r6uq6EVfdVLS/GrgqqA2ViOtw9puA+zfbAKCq41T1DFy1UCLuuKCqy1S1D6767+/AuyKSepSxmMNgicAciRrAz8CvItIKuCkM3/kB8DsRuUxEkoChQL0QxTgBuFNEmohIHeD+8jZW1S3AF8DrwDJVXeFblQJUAfKAAhG5FDj/MGJ4SESOFXefxRC/ddVxJ/s8XE4chCsRFNkCNC1qHA/gLeBGEWknIim4E/LnqlpmCeswYu4hIl19330vrl1nroi0EpFzfd+31/cowP2Aa0Wkrq8E8bPvtxUeZSzmMFgiMEfibuB63B/5P3FXxCHlO9leDTwHbAeOB77D3fcQ7BhH4uryf8A1ZE6sxGfexDX+vukX807gLmASrsG1Fy6hVcZjuJLJGuC/wL/99vs98ALwjW+bkwD/evWPgRXAFhHxr+Ip+vyHuCqaSb7Pp+PaDY6Kqi7CHfORuCTVDejhay9IAZ7GtetsxpVAHvZ9tDuwRFyvtGeBq1X1wNHGYypPXFWrMdFFRBJxVRG9VPVzr+MxJppZicBEDRHpJiI1fdULj+B6onzjcVjGRD1LBCaanAmsxlUvdAMuV9WyqoaMMZVkVUPGGBPnrERgjDFxLuoGnatbt65mZmZ6HYYxxkSV+fPnb1PVgF2uoy4RZGZmkpOT43UYxhgTVUSkzDvkrWrIGGPinCUCY4yJc5YIjDEmzkVdG0EgBw8eJDc3l3379nkdStRLTU2ladOmJCeXNUyNMSbWxEQiyM3NpUaNGmRmZuIGpTRHQlXZvn07ubm5NG/evOIPGGNiQkxUDe3bt486depYEjhKIkKdOnWsZGVMpMnOhsxMSEhwz9nZFX3isMREiQCwJBAkdhyNiTDZ2TB4MOzZ496vXeveA/Q76kFjgRgpERhjTMx68MHiJFBkzx4YNixoXxEzJQJjjIl6hYWwdCnMnQtz5rjH+vWBt123LmhfG5clgmBXt+3cuZOXX375sD/XvXt3du7cedifGzBgABMnVmauFGNMRMvLgw8+gEcegQsvhFq1oE0buOEGmDABGjaEmjUDfza9ohlUKy/uSgShqG4rSgS33nprieUFBQUkJiaW+blp06Yd2RcaY6LPgQOwYEHx1f7cubBqlVuXmAht20LfvtClC3TuDC1buqvV0ictgLQ0GD48aKHFZCLo2vXQZb17w623ll3dNnSoSwTbtkGvXiXXf/pp+d/3wAMPsGrVKjp06EBycjLVq1enUaNGLFiwgMWLF3P55Zezfv169u3bx9ChQxnsyzxF4ybt3r2biy++mDPPPJOvvvqKJk2aMHnyZKpWrVrhb50xYwb33HMP+fn5dOrUiZEjR5KSksIDDzzAlClTSEpK4qKLLuLZZ5/lnXfe4U9/+hOJiYnUrFmT2bNnV7h/Y8wRUHVXmUUn/Dlz4LvvYL9v+ozGjd0J/6ab3Em/Y0eoVi3wvoquUIcNc9VB6ekuCQSpoRhiNBGUJ7eM6bm3bz/yfT755JP8+OOPLFiwgE8//ZRLLrmEH3/88be++GPGjKF27drs3buXTp06ceWVV1KnTp0S+1ixYgVvvfUWr7zyCr179+bdd9+lf//+5X7vvn37GDBgADNmzKBly5Zcd911jBw5kuuuu45JkyaxdOlSROS36qcnnniC6dOn06RJkyOqkjLGlGHXLpg3r+TV/pYtbl3Vqu5Ef/vt7qTfpQs0bXp4++/XL6gn/tJClghEZAxwKbBVVU8OsL4fcL/v7W7gFlVdGIzvLu8KPj3dJerSMjLcc926FZcAKnLqqaeWuCHrhRdeYNKkSQCsX7+eFStWHJIImjdvTocOHQDo2LEja9asqfB7li1bRvPmzWnZsiUA119/PS+99BJDhgwhNTWVQYMGcckll3DppZcCcMYZZzBgwAB69+7NFVdccXQ/0ph4kJ196JV4nz6wZEnJk/6PP7pSALgqnd//vvik37YtRPid+qEsEYwFXgT+Xcb6/wHnqOoOEbkYGA10DmE8gPt3DHF1G9X8iniffvopn3zyCV9//TVpaWl07do14A1bKSkpv71OTExk7969FX5PWbPLJSUl8c033zBjxgzefvttXnzxRWbOnMmoUaOYO3cuU6dOpUOHDixYsOCQhGSM8QnUoHjddXDjjcVVPLVquRP+FVe4k/6pp0Lt2t7FfIRClghUdbaIZJaz/iu/t3OAwywrHZlQVLfVqFGDXbt2BVz3888/U6tWLdLS0li6dClz5sw58i8q5aSTTmLNmjWsXLmSE044gTfeeINzzjmH3bt3s2fPHrp3706XLl044YQTAFi1ahWdO3emc+fOvP/++6xfv94SgTGB7N0Ld955aINiYaG7uh892p34W7SAGLgJM1LaCG4E/lvWShEZDAwGSA9Cl6lgV7fVqVOHM844g5NPPpmqVavSoEGD39Z169aNUaNG0a5dO0488US6dOkStO9NTU3l9ddf56qrrvqtsfjmm2/mp59+omfPnuzbtw9VZcSIEQDce++9rFixAlXl/PPPp3379kGLxZioV1AAs2a5ksC777p6/0B+/dWVDGJISCev95UIPgjURuC3zbnAy8CZqlphk21WVpaWnqFsyZIltGrV6uiCNb+x42nihqrr0jluHLz1FmzaBDVquK6DU6fC1q2HfiYjAyrRhhdpRGS+qmYFWudpiUBE2gGvAhdXJgkYY0xQrFkDb77pEsCSJa665+KLoX9/uPRS19MnDP33I4VniUBE0oH/ANeq6nKv4ohkt912G19++WWJZUOHDmXgwIEeRWRMFNu+Hd55x53gv/jCLTvzTBg1ypUASreXhaH/fqQIZffRt4CuQF0RyQUeA5IBVHUU8ChQB3jZN+JlflnFlnj10ksveR2CMdFt7154/3138v/vf+HgQWjd2p3Q+/Z1Y8yUJ8T99yNFKHsNXVPB+kHAoFB9vzEmTgVq9G3cGO64w1X9tG8fEz19gilSeg0ZY8yRC9Toe8wxrsqnXz837kw5437FO0sExpjoFajRt3t3d/IvavQ1FbJEYIyJLoEafc86yzX6XnVVVN7Z67W4nI8g1PN/VqR69eplrluzZg0nn1zmbRfGxIfSf6Ovv+7G5+/ZExo1gltugZ9+gr/+1ZUKZs92I3laEjgi8VciCMP8n8aYoxDob/SGG9zrxo2Lx4y3Rt+gib1EcOedrtGoLHPmFA8YVWTPHjeQ1CuvBP5Mhw7wj3+Uucv777+fjIyM3yamefzxxxERZs+ezY4dOzh48CB/+ctf6Nmz52H9lH379nHLLbeQk5NDUlISzz33HOeeey6LFi1i4MCBHDhwgMLCQt59910aN25M7969yc3NpaCggEceeYSrr776sL7PmIgQaNIQgPr1XX9+a/QNuthLBBUpnQQqWl4Jffr04c477/wtEUyYMIEPP/yQu+66i2OOOYZt27bRpUsXevTogRzGFUzRfQQ//PADS5cu5aKLLmL58uWMGjWKoUOH0q9fPw4cOEBBQQHTpk2jcePGTJ06FXCD3RkTVXbuhJEjy56jNy/PkkCIxF4iKOfKHXD1jWVNSHCEExGccsopbN26lY0bN5KXl0etWrVo1KgRd911F7NnzyYhIYENGzawZcsWGjZsWOn9fvHFF9x+++2AG2k0IyOD5cuXc9pppzF8+HByc3O54ooraNGiBW3btuWee+7h/vvv59JLL+Wss846ot9iTNht3Oj+bkeNcn3+U1MhwFDtwZyj15QUf43Fw4e78UL8BWH8kF69ejFx4kTGjx9Pnz59yM7OJi8vj/nz57NgwQIaNGgQcB6C8pQ1IGDfvn2ZMmUKVatW5fe//z0zZ86kZcuWzJ8/n7Zt2/Lggw/yxBNPHNXvMSbkli2DQYOgeXP4+9/hkkvg22/h1VdD8jdqyhZ7JYKKhGj8kD59+vCHP/yBbdu28dlnnzFhwgTq169PcnIys2bNYm2gUkgFzj77bLKzsznvvPNYvnw569at48QTT2T16tUcd9xx3HHHHaxevZrvv/+ek046idq1a9O/f3+qV6/O2LFjj+r3GBMy33wDTz0FkyZBSopLBnffDccd59afcop7joMxfiJF/CUCCMn4IW3atGHXrl00adKERo0a0a9fPy677DKysrLo0KEDJ5100mHv89Zbb+Xmm2+mbdu2JCUlMXbsWFJSUhg/fjzjxo0jOTmZhg0b8uijjzJv3jzuvfdeEhISSE5OZuTIkUH9fcYcFVX46COXAGbNgmOPhYcecsM+1K9/6PZxMsZPpAjpfAShYPMRhJ4dTxM0+fkwcaJLAAsWuO6ff/yj6x5ao4bX0cWViJ2PwBgTo/buhbFj4dlnYfVqOPFEeO01d5XvNz+3iQyWCDzyww8/cO2115ZYlpKSwty5cz2KyJgg2LEDXn4ZXnjBze7VubNrCO7Rw90lbCJSzCQCVT2sPvpea9u2LQvKu/HNI9FWVWgixIYNMGIE/POfsHs3dOsGDzwAZ59td/9GgZhIBKmpqWzfvp06depEVTKINKrK9u3bSU1N9ToUEy2WLoVnnoE33nDzAPTpA/fd54Z/MFEjJhJB06ZNyc3NJS8vz+tQol5qaipNmzb1OgwT6ebOdQ3A773n6vwHD3ZdQJs39zoycwRiIhEkJyfT3P4DGhNaqvDhhy4BfPYZ1Krl+vrffnvgLqAmaljrjTGmWKAh2vPz3eQvp5ziJn1ZuRKee84N1fLnP1sSiAExUSIwxgRBoOGfBw50I/pu2watWrl5Afr2hSpVvI3VBJUlAmOMM2zYocM/HzwIv/zi2gIuu8y6gMYoSwTGGGfdusDLDx50M4OZmGXp3Rjj1KkTeLkN/xzzQpYIRGSMiGwVkR/LWC8i8oKIrBSR70Xkd6GKxRhTjvx8uOce1w5QuurHhn+OC6EsEYwFupWz/mKghe8xGLDhMo0Jt82b4YIL3DAQt90GY8a4SZpE3PPo0TYKaBwIWRuBqs4WkcxyNukJ/FvdmAZzRORYEWmkqptCFZMxxs+XX8JVV7kpIt94A/r3d8uvv97buEzYedlG0ATwn5w017fsECIyWERyRCTH7h425iipukHhunaFatVgzpziJGDikpeJINCgQAFHPFPV0aqapapZ9erVC3FYxsSw3btdVc/Qoe7msHnzoF07r6MyHvMyEeQCzfzeNwU2ehSLMbFv+XLo0gXGj4e//tVNFXnssV5HZSKAl4lgCnCdr/dQF+Bnax8wJkQmTYKsLNiyBaZPhwcftJvDzG9C1lgsIm8BXYG6IpILPAYkA6jqKGAa0B1YCewBBoYqFmPiVn6+u2P46afh1FPdtJHNmlX8ORNXQtlr6JoK1itwW6i+35i4t2ULXHONmyz+5pvhH/+waSJNQDbEhDGx6OuvXdfQ7dvhX/+C667zOiITwayS0JhYogovvgjnnOOu/r/+2pKAqZAlAmNixa+/wrXXuolifv97yMmBDh28jspEAUsExsSCFStc19A333STxUye7GYQM6YSrI3AmGg3ebKr/klKclNJXnSR1xGZKGMlAmOiVX6+ux/g8suhZUv49ltLAuaIWInAmGiUl+e6hs6Y4aaXfP55SE31OioTpSwRGBNt5s6FXr1cMhgzxs0rbMxRsKohY6KFKowcCWed5doDvvrKkoAJCksExkSDPXtgwAC49VY3kcz8+fA7m9TPBIclAmMi3apVcNppbvKYxx+HDz6A2rW9jsrEEGsjMCaSvf++u0ksIQGmTYNu5c3+asyRsRKBMZGooAAefhh69IDjj3dVQZYETIhYIjAmUmRnQ2amu/qvXh2GD4cbb3RzCzdv7nV0JoZZ1ZAxkSA7290PsGePe79vH1SpAueea/cHmJCzEoExkWDYsOIkUOTAAbfcmBCzRGCM1/bvh7VrA69bty68sZi4ZInAGC+tXw9nn132+vT08MVi4pYlAmO8MmsWdOwIS5bAnXdCWlrJ9WlprsHYmBCzRGBMuKnCc8/BhRdCnTrwzTcwYgSMHg0ZGSDinkePhn79vI7WxAHrNWRMOO3eDYMGwfjxcMUVMHYs1Kjh1vXrZyd+4wkrERgTLitWuKEi3nkHnnwSJk4sTgLGeCikiUBEuonIMhFZKSIPBFifLiKzROQ7EfleRLqHMh5jPPPBB9CpE2za5GYRu/9+VwVkTAQIWSIQkUTgJeBioDVwjYi0LrXZw8AEVT0F6AO8HKp4jPFEYSE89hhcdpkbKiInx7UNGBNBQlkiOBVYqaqrVfUA8DbQs9Q2Chzje10T2BjCeIwJrx07XAJ44gm4/nr44gs3hIQxESaUjcVNgPV+73OBzqW2eRz4SERuB6oBF4QwHmPC5/vvXWPwunXw8stw881WFWQiVihLBIH+12up99cAY1W1KdAdeENEDolJRAaLSI6I5OTl5YUgVGOC6K23XKPwnj3w6adwyy2WBExEC2UiyAWa+b1vyqFVPzcCEwBU9WsgFahbekeqOlpVs1Q1q169eiEK15ijdPAg/PGP0Levmz3s22/h9NO9jsqYCoUyEcwDWohIcxGpgmsMnlJqm3XA+QAi0gqXCOyS30SfLVtcI/CIEXDHHTBzJjRs6HVUxlRKyNoIVDVfRIYA04FEYIyqLhKRJ4AcVZ0C3A28IiJ34aqNBqhq6eojYyLbnDnQqxf89JObTrJ/f68jMuawhPTOYlWdBkwrtexRv9eLgTNCGYMxIaPqhoG4/XZo2hS++go6dPA6KmMOm91ZbMyR2LcP/vAH1xvovPPc/QGWBEyUskRgzOFatw7OOgtee83NKzx1KtSu7XVUxhwxG3TOmMMxcyZcfbWbTOa996Bn6XskjYk+ViIwpjJU4dlnXc+gevVg3jxLAiZmWInAmIrs3g033OBGDe3VC8aMsVFDTUyxEoEx5Vm+HDp3hnffhaefhgkTLAmYmGMlAmPKMmUKXHstJCfD9OlwgQ2FZWKTlQiMKa2gAB591LUBtGgB8+dbEjAxzRKBMdnZbnjohARo1syNE/TnP8PAgW7o6IwMryM0JqSsasjEt+xsGDzYjRQKkJvrHgMHuvsEbNRQEwesRGDi27BhxUnA38yZlgRM3LBEYOLbunWHt9yYGGSJwMS3soaGSE8PbxzGeMgSgYlf48bB9u2ukdhfWhoMH+5NTMZ4wBKBiU/jx7sJ5c89F1591fUMEnHPo0dDv35eR2hM2FivIRN/Jk1yJ/rTT4f334dq1VwvIWPiVKVKBCIyVESOEec1EflWRC4KdXDGBN0HH7jRQzt1gmnTXBIwJs5VtmroBlX9BbgIqAcMBJ4MWVTGhMJHH8GVV0K7dvDf/9qYQcb4VDYRFHWo7g68rqoL/ZYZE/lmzXJDRrRq5RLCscd6HZExEaOyiWC+iHyESwTTRaQGUBi6sIwJos8/h0svheOPh48/ttnEjCmlso3FNwIdgNWqukdEauOqh4yJbHPmQPfubnL5Tz5xk8oYY0qobIngNGCZqu4Ukf7Aw8DPoQvLmCCYPx+6dYMGDdyQEQ0beh2RMRGpsolgJLBHRNoD9wFrgX+HLCpjjtbChW5ayVq1XBJo0sTriIyJWJVNBPmqqkBP4HlVfR6osMuFiHQTkWUislJEHihjm94islhEFonIm5UP3ZgyLFrk5g+oVs0lARsuwphyVbaNYJeIPAhcC5wlIolAcnkf8G3zEnAhkAvME5EpqrrYb5sWwIPAGaq6Q0TqH8mPMOY3y5bB+ee7WcVmzoTmzb2OyJiIV9kSwdXAftz9BJuBJsAzFXzmVGClqq5W1QPA27gShb8/AC+p6g4AVd1a6ciNKW3lSjjvPFCFGTPc7GLGmApVKhH4Tv7ZQE0RuRTYp6oVtRE0Adb7vc/1LfPXEmgpIl+KyBwR6RZoRyIyWERyRCQnLy+vMiGbeLNmjUsC+/e73kGtWnkdkTFRo7JDTPQGvgGuAnoDc0WkV0UfC7BMS71PAloAXYFrgFdF5JA7fVR1tKpmqWpWPev+Z0pbv94lgV273H0Cbdt6HZExUaWybQTDgE5FVTciUg/4BJhYzmdygWZ+75sCGwNsM0dVDwL/E5FluMQwr5JxmXi3aZNrE9i+3ZUETjnF64iMiTqVbSNIKFV/v70Sn50HtBCR5iJSBegDTCm1zXvAuQAiUhdXVbS6kjGZeLd1q0sCGze6sYM6dfI6ImOiUmUTwYciMl1EBojIAGAqMK28D6hqPjAEmA4sASao6iIReUJEevg2mw5sF5HFwCzgXlXdfiQ/pDzZ2ZCZ6eYfycx0702U27bNdRFds8aNInr66V5HZEzUEnd7QCU2FLkSOANX9z9bVSeFMrCyZGVlaU5OTqW3z86GwYNLzk+elmZzj0S1HTtcm8DSpW5Y6fPP9zoiYyKeiMxX1ayA6yqbCCLF4SaCzExYu/bQ5RkZ7mLSRJmff3Z3DC9cCJMnuyEkjDEVKi8RlNtYLCK7OLSnD7hSgarqMUGIL6TWrQu8fO1ad0F50knhjccchV273ABy330H//mPJQFjgqTcNgJVraGqxwR41IiGJADljy7QqhX06FH2ehNB9uyByy6DuXPh7bfda2NMUMT8nMXDhwduI3jqKSgshKpV3bKDB11V8wUXwFVX2f1IEWXfPjepzOefw7hxbpYxY0zQxHwiKGoQHjbMVROlp7vkULqheMsWNzLB44/DY4/BySe7hDBwIDRrdshuTbjs3w9XXOGGjHj9dbjmGq8jMibmxHxj8eHauBHefRfeeQe++AI+/RTOPts1LO/dayWFsDpwwGXjKVPglVdg0CCvIzImapXXWFzZ+wjiRuPGcPvtMHs25ObCGWe45f/4B7Ru7UoKf/oTLF5c/n7MUcrPh759XRJ48UVLAsaEkCWCcjRuDImJ7vX998P//R/UqeMSQZs2cNZZrjrJBFlBAVx3nSuaPfcc3Hab1xEZE9Nivo0gWBo1giFD3GPTJneO2rMHRFwy6NkTfvc7V5PRpo3X0UaxwkJ39f/WW/C3v8Fdd3kdkTExz9oIgmDnTrj8cledpOraEXr3huuvt3lRDosq3Hyzu+37T3+CRx/1OiJjYoa1EYTYsce6RuUNG1x1dv368MQTMM83huqWLW72RBvzqByqcMcdLgk89BA88ojXERkTN6xEECKbN0PNmu4+hb/9zZ3biqqRisT9mEfZ2cX9eqtXd3cO3303PPOMO1jGmKCxEoEHGjYsvlnthhugVq1DG5b37HHnwR074rDRuWg0wLVr3Y/ftQuSktx8ApYEjAkrSwRh0KCBa0cIZN06uOQS10Opb1/XXX7VqhhPDAUFcN99JW/3BtdldNgwb2IyJo5Zr6EwSU8PPApqerq7MP7oI5g1y3WWAdd78l//cq83bnSJIipt2QI//FDysWiRu9xgLL0AABPdSURBVDsvkLJGCTTGhIwlgjApa8yjouEuBgxwpYBly1xCKBrWIi8PmjRxvY/OPdcNw3/uuRGYGPbscSf40if9rX4T29Wv7+YTvukmeOMNN71kaeWNEmiMCYn4SAT+jZJlDTYUYpUZ80jEDYvtPzR2UhI8/7xLDpMmwZgxbnl2tqtK+uUXNyZb/fph+iEFBa7uqvQJf+XK4vqsqlXdzRSXXOJO/O3auWf/ILOyys6MxpjwUtWoenTs2FEPy7hxqmlpqu405R5paW55uI0bp5qRoSring8zhvx81fnzVZ99VnXtWrfstdfcT2rTRnXIENV331Xdti1IcWzerPrxx6rPPac6cKBqx46qVasWH0cR1ZYtVa+8UvXxx92XL1/uAq2MozwexpjKA3K0jPNq7HcfLWuKstq14e9/d2NIJCS451C+njwZ7rmnZN141aowahT07++2OQLLl7s5WmbNcoPkFd3tvHUr1K3rBsurVct1ZQUgO5v8GwaTdKD4Sjy/SlWSHn7Q1UF9/33xVX5eXvEXNWjgrur9H61bu6t4Y0zEi+upKklIiI4uOCKuHigpySUP/+eKlvleFyYm8cuvifzyaxLpx7llX89LYsPmRGrUTqJewyTarJhEysFfy44jLc1V65Q+6Yet7skYEwpHPFVlTCiru06TJm6ik8JCV+9dUBDa1zfdVHaMjz/uuk4WFLhn/9eHsSwhP59jU/dzbPIe2OqWtauWz/F189n3awEHF+dThcBJQBFkxXI47rgjLp0YY6JT7CeC8qYoC+dAQH/9a+CElJHhZsIJkWq+B8Cvv8La6plkcmgca0kn/bgTLAcYE4di/8++Xz83jkNGhqt+ycjwZlyH4cMPrU8Pcy+ZatXguTrD+ZWScfxKGg8xnOOPd/nKv2nAGBP7QpoIRKSbiCwTkZUi8kA52/USERWRgPVXR61fP9dqWljonr0Y3CdCElLn5/sxJHk0a8igEGENGQxJHk2dIf047jjXvXXTJrftzp3ukBljYlvIGotFJBFYDlwI5ALzgGtUdXGp7WoAU4EqwBBVLbclOFoGnYtk5d1WsXaty1EA117reiL94Q9uvKSGDb2L2RhzdLwadO5UYKWqrlbVA8DbQM8A2/0ZeBrYF8JYjJ/yCkhFSQDcHAvNm7uk0awZ9OoFn30W7miNMaEWykTQBFjv9z7Xt+w3InIK0ExVPyhvRyIyWERyRCQnzyqww+bKK2HmTDfsxdChbs6FqVPdusJCN4yQMSb6hTIRBBpL+Ld6KBFJAEYAd1e0I1UdrapZqppVr169IIZoKqNlS3j2WcjNdfMqAHzyCTRt6qbm/OQTa0swJpqFMhHkAs383jcFNvq9rwGcDHwqImuALsCUkDUYm6OWmupmYwOXHO64w5UYLrzQvX/qqbIHFTXGRK5QJoJ5QAsRaS4iVYA+wJSilar6s6rWVdVMVc0E5gA9KmosNpEhM9ON0LFhg2t8btLETdOZnOzWF7VBGGMiX8gSgarmA0OA6cASYIKqLhKRJ0SkR6i+14RXaqobBfWzz9zwRElJcPAgnH46nHiim3XSmnWMiWyxP9aQCbuDB2H8eHebxOefQ5UqcMUV8OCDbkRqY0z42ZzFJqySk92AqrNnu7lqbrkFPvyweI6arVth2zZXpZSZ6YY2ysx0740x4WclAhMWe/dCSoo76d99t5tsB9yYeUXS0rwZ/cOYeGAlAuO5qlWLBzW98UZ30vdPAuDGBbS5640JP0sEJuxat4bduwOvs7nrjQk/SwTGE2XNUd+4sd2LYEy4WSIwnihrVO6UFOjcGRYvDvw5Y0zwWSIwnihrVO4XX4TNmyEry72Psr4MxkQl6zVkIs7mzXDddfDxx27E01dfhZo1vY7KmOhmvYZMVGnY0N138PTTsGJF8bAVxpjQsERgIlJCAtx7L8yb59oOdu+GF144tMupMeboWSIwEa2oNPDWW25OhPPPd8NhG2OCxxKBiQqDBsHYsZCTA+3bw+TJXkdkTOywRGCigghcfz18+63rYXT55a6qyBhz9CwRmKjSsiV8/TXcfz/0sMHMjQkKSwQm6qSkwJNPuhFLVaFPH7vnwJijYYnARLVff4Xt2+Gmm6B3b9ixw+uIjIk+lghMVKteHaZPd/Mlv/cedOgAX33ldVTGRBdLBCbqJSTAfffBF19AYiJcfTXs3+91VMZEjySvAzAmWDp3hu++g9WrXTtCfr6bL7lRI68jMyayWYnAxJSaNeGUU9zrv/0NTj4ZpkzxNiZjIp0lAhOzrr7a3XPQsyfcfjvs2+d1RMZEJksEJmYV3XPwxz+64a1PPRWWLPE6KmMijyUCE9NSUuDvf4epU+Gnn1x3U2NMSSFNBCLSTUSWichKEXkgwPo/ishiEfleRGaISEYo4zHxq3t3WLXKTXgD8PrrsHOntzEZEylClghEJBF4CbgYaA1cIyKtS232HZClqu2AicDToYrHmJQU97xqFQweXHzPQXa2u0s5IcE9Z2d7GaUx4RfKEsGpwEpVXa2qB4C3gZ7+G6jqLFXd43s7B2gawniMAeD44+HLL909B2eeCQMHwtq1boiKtWtdkrBkYOJJKBNBE2C93/tc37Ky3Aj8N9AKERksIjkikpOXlxfEEE28OvVUd89B1apw8GDJdXv2wLBh3sRljBdCmQgkwLKAw4KJSH8gC3gm0HpVHa2qWaqaVa9evSCGaOLZMcfA3r2B161bF95YjPFSKBNBLtDM731TYGPpjUTkAmAY0ENVbWAAE1bp6YGX160b3jiM8VIoE8E8oIWINBeRKkAfoMQ9niJyCvBPXBLYGsJYjAlo+HA3J7I/ETc0xWWXuYZlY2JdyBKBquYDQ4DpwBJggqouEpEnRKRoSpFngOrAOyKyQERsMAATVv36ubkMMjJcAsjIcF1Ln3kGPv0U2rSBl17yOkpjQks0ymbzyMrK0pycHK/DMHFg40a491649lro1s0NYpeY6BKGMdFGROaralagdXZnsTFlaNzYdSPt1s29f/RRuPhiWL7c27iMCTZLBMZUUtOmbuyik0+GBx+E3bu9jsiY4LBEYEwl3XqrKw307evmTG7VCmbN8joqY46eJQJjDkODBjB2rLszuVEjKLqtJcqa2owpwWYoM+YInH46zJ1b3HB8/fUuKTz2mLtRzZhoYiUCY45QURIoKHBDVYwYASeeCOPGWQnBRBdLBMYcpcRE+Oc/XQkhPd11Nz37bLsZzUQPSwTGBEmnTq5X0auvwubNUKOG1xEZUzmWCIwJooQEuPFGWLoU6teHwkI3Z/KYMe61MZHIEoExIZCY6J5/+gm2bXPJ4YwzYP58b+MyJhBLBMaEUN268Pnn8K9/wf/+56qPbr4ZfvnF68iMKWaJwJgQS0iA666DZctg6FA3mF2VKl5HZUwxSwTGhEnNmq6L6YIFkJrqZkL7f/8P5sxx623uZOMVu6HMmDBLTXXPK1fCN9/Aaae57qbz5hXPmFY0dzK4obKNCSUrERjjkXbtXO+i++6D2bMPnTbT5k424WKJwBgP1agBTz1V9hwHa9fCK6+4ksOePeGNzcQPSwTGRICy5k4WcVVEnTu7pHH33W65KkyfDhs22HAW5uhZG4ExEWD4cHfC97/qT0tzQ1ecdhp8/z0sXAgdOrh1mzYVT5hTp46rZmrf3g2R3alT+OM30c0SgTERoKhBeNgwWLfOlRCGDy9efvzxrodRkTp13P0JCxe6XkgLF7qk0bGjSwTffee6rLZvX/LRoEH4f5uJfDZnsTExoqDAPapUgZwcNyT2woWu+qjIrFnQtSssWuTWtW8PLVtCcrJbn51ddjIy0a28OYutRGBMjEhMLB7aIisLpk51r7dvL65aat/eLZs0CR55xL1OSYHWraF6dZdArAtr/LESgTFx6OBBd6dzUbXSwoUwc6YrUZSWmOjaJpo0gcmT3bKJE12poXbt4ke9em4+hmCwkklJwTgeViIwxpSQnAwnn+we/fu7ZQll9CEsKHBtC/7XjGPHFpc4ihx/vLtJDqBHD5dk/BNF27auugpgyhSXjGrXdu0dRdukpbmTnn/DuZclk0hISOE4HiEtEYhIN+B5IBF4VVWfLLU+Bfg30BHYDlytqmvK26eVCIwJjcxMd5IpLSMD1qwpuaywEHbtcqOrFj1U4aKL3PoRI1x1lP/6E0+E//zHrW/dGpYsKbnPCy+Ejz4qO45atdx+AB5+2MVQpYqr2kpJcdVe55/v1o8fX7yu6Dk93f2WwkL3e/w/W6WKe/gnw9InYHCJavTo8k/AhYWuei0/3z0OHnTPtWpBtWqwe7cbgLBofdGjTRuXDDdtcneZF33utttc9V5l/l3KU16JAFUNyQN38l8FHAdUARYCrUttcyswyve6DzC+ov127NhRjTHBN26calqaqjulu0damlsebBs2qC5cqDprlurEiaqjR6u+955bJ1IyBv9HkXr1VJOSSq676Sa3Lj8/8Gfvvtut//nnwOsfe8yt37hRtXZt1YSEwNs1bqxav77bpkYN1apVVZOTVV97zX1+zpzAnxs/3q3/+OPA66dNc+snTSr79/s/RA7vmAM5WsZ5NZRVQ6cCK1V1tS8bvQ30BBb7bdMTeNz3eiLwooj7eSGMyxgTQEVdWIOpcWP3CCQ9veySSZGtW91zYSEcOAD79xc3lCckwOLFbtn+/cXrmzVz61NSXNVW0fKibc48s3h9377w4ouB49u0CW66CZKS3CM52T23bVsc/1NPFS8v2ibLdy3erp1rYylaV/Qoasg/5xw3b0XR8gsvhI0bAx+noCkrQxztA+iFqw4qen8t8GKpbX4Emvq9XwXUDbCvwUAOkJOenn54adAYE1XCWTIpT0ZG4CvxjIzwxhGs40E5JYJQDjERaPSU0lf6ldkGVR2tqlmqmlWvXr2gBGeMiUz9+rl6+IwMN8RGRkbF9fKhMHy4axPwl5bmlodTOI5HKKuGcoFmfu+bAqULOEXb5IpIElAT+CmEMRljokC/ft53Fw1nVVllYgnl94YyEcwDWohIc2ADrjG4b6ltpgDXA1/jqpJm+oowxhjjuUhISOEQskSgqvkiMgSYjutBNEZVF4nIE7i6qinAa8AbIrISVxLoE6p4jDHGBBbSG8pUdRowrdSyR/1e7wOuCmUMxhhjymfzERhjTJyzRGCMMXHOEoExxsS5qBt9VETygAD3HUaVusA2r4OIIHY8SrLjUcyORUlHczwyVDXgjVhRlwhigYjkaFmDP8UhOx4l2fEoZseipFAdD6saMsaYOGeJwBhj4pwlAm+M9jqACGPHoyQ7HsXsWJQUkuNhbQTGGBPnrERgjDFxzhKBMcbEOUsEYSQizURklogsEZFFIjLU65i8JiKJIvKdiHzgdSxeE5FjRWSiiCz1/R85zeuYvCQid/n+Tn4UkbdEJNXrmMJJRMaIyFYR+dFvWW0R+VhEVvieawXjuywRhFc+cLeqtgK6ALeJSGuPY/LaUGBJhVvFh+eBD1X1JKA9cXxcRKQJcAeQpaon40YwjrfRiccC3UotewCYoaotgBm+90fNEkEYqeomVf3W93oX7g+9ibdReUdEmgKXAK96HYvXROQY4Gzc0Oyo6gFV3eltVJ5LAqr6Jq1K49CJrWKaqs7m0Im6egL/8r3+F3B5ML7LEoFHRCQTOAWY620knvoHcB9Q6HUgEeA4IA943VdV9qqIVPM6KK+o6gbgWWAdsAn4WVU/8jaqiNBAVTeBu7AE6gdjp5YIPCAi1YF3gTtV9Rev4/GCiFwKbFXV+V7HEiGSgN8BI1X1FOBXglTsj0a+uu+eQHOgMVBNRPp7G1XsskQQZiKSjEsC2ar6H6/j8dAZQA8RWQO8DZwnIuO8DclTuUCuqhaVECfiEkO8ugD4n6rmqepB4D/A6R7HFAm2iEgjAN/z1mDs1BJBGImI4OqAl6jqc17H4yVVfVBVm6pqJq4RcKaqxu0Vn6puBtaLyIm+RecDiz0MyWvrgC4ikub7uzmfOG4891M0zzu+58nB2GlIp6o0hzgDuBb4QUQW+JY95JvS05jbgWwRqQKsBgZ6HI9nVHWuiEwEvsX1tvuOOBtuQkTeAroCdUUkF3gMeBKYICI34pJlUKb6tSEmjDEmzlnVkDHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoExxsQ5SwTG+IhIgYgs8HsE7c5eEcn0H0XSmEhi9xEYU2yvqnbwOghjws1KBMZUQETWiMhTIvKN73GCb3mGiMwQke99z+m+5Q1EZJKILPQ9ioZGSBSRV3xj7H8kIlV9298hIot9+3nbo59p4pglAmOKVS1VNXS137pfVPVU4EXcqKn4Xv9bVdsB2cALvuUvAJ+panvceEGLfMtbAC+pahtgJ3Clb/kDwCm+/dwcqh9nTFnszmJjfERkt6pWD7B8DXCeqq72DRq4WVXriMg2oJGqHvQt36SqdUUkD2iqqvv99pEJfOybUAQRuR9IVtW/iMiHwG7gPeA9Vd0d4p9qTAlWIjCmcrSM12VtE8h+v9cFFLfRXQK8BHQE5vsmYjEmbCwRGFM5V/s9f+17/RXF0yf2A77wvZ4B3AK/zcl8TFk7FZEEoJmqzsJN0nMscEipxJhQsisPY4pV9RsVFtz8wUVdSFNEZC7u4uka37I7gDEici9udrGi0UKHAqN9I0QW4JLCpjK+MxEYJyI1AQFG2BSVJtysjcCYCvjaCLJUdZvXsRgTClY1ZIwxcc5KBMYYE+esRGCMMXHOEoExxsQ5SwTGGBPnLBEYY0ycs0RgjDFx7v8DRAkD1q/vJBAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xUZdbA8d8BAiEKogiKhCSoLB1EIsXKWlZQFtRVAdFXVOC1rb2wYEEU29qWd1kxKjaCiqwCNlQUxYJKUESkiZQQaaFKJ8B5/3gmZJJMwiSZyZ3MPd/P535m5t47MyeTmXvuU+7ziKpijDHGv6p5HYAxxhhvWSIwxhifs0RgjDE+Z4nAGGN8zhKBMcb4nCUCY4zxOUsEJuJEpLqIbBORlEju6yUROV5EIt7XWkTOFpHlQY8Xichp4exbjvd6QUSGlvf5Jn7V8DoA4z0R2Rb0MAnYDewLPP5fVc0sy+up6j7g0Ejv6weq2jwSryMiA4HLVbVb0GsPjMRrm/hjicCgqgcOxIEzzoGqOq2k/UWkhqrurYzYjDHRZ1VD5qBE5CEReVNEXheRrcDlItJVRL4Vkc0islpERolIQmD/GiKiIpIWeDwusP1DEdkqIjNFpGlZ9w1s7yEii0Vki4j8n4h8LSIDSog7nBj/V0SWiMgmERkV9NzqIvK0iGwQkd+A7qV8PveIyBtF1o0WkacC9weKyILA3/Nb4Gy9pNfKEZFugftJIvJaILZfgI4h3ndp4HV/EZFegfVtgX8DpwWq3dYHfbbDg55/beBv3yAik0SkUTifTRk/52JVaiLyVfD/LPA+CwN/xzwRaV/Se5koUVVbbDmwAMuBs4usewjYA/wVd/JQGzgJ6IwrVR4LLAZuDOxfA1AgLfB4HLAeSAcSgDeBceXYtyGwFegd2HYbkAcMKOFvCSfGycBhQBqwMf9vB24EfgGSgfrADPdzCfk+xwLbgEOCXnsdkB54/NfAPgKcCewE2gW2nQ0sD3qtHKBb4P4TwOfA4UAqML/IvpcCjQL/k8sCMRwV2DYQ+LxInOOA4YH7fwnEeAKQCPwH+Cycz6aMn/PxRT834Kv8/xnQD1iJS3IC/Alo4vXvwG+LlQhMuL5S1XdVdb+q7lTVWar6naruVdWlQAZwRinPn6iqWaqaB2TiDkBl3bcnMEdVJwe2PY1LGiGFGeMjqrpFVZfjDrr573Up8LSq5qjqBuDRUt5nKTAPl6AAzgE2q2pWYPu7qrpUnc+AT4GQDcJFXAo8pKqbVHUF7iw/+H0nqOrqwP9kPC6Jp4fxugD9gRdUdY6q7gKGAGeISHLQPiV9NoWU47sQbCDwqKrODnw+i1V1ZZjPNRFiicCEq9CPU0RaiMj7IrJGRP4ARgBHlvL8NUH3d1B6A3FJ+x4THIeqKu4MOqQwYwzrvYAVpcQLMB53dgvu7PxAA7uI9BSR70Rko4hsxp2Nl/ZZ5WtUWgwiMkBEfgpUyWwGWoT5uuD+vgOvp6p/AJuAxkH7hPU/K8d3IVgT4Lcw9zVRYonAhKto18nncGfBx6tqXeA+XNE+mlbjqmoAEBGh8IGrqIrEuBp3kMp3sO6tbwJnB86oe+MSAyJSG5gIPIKrtqkHfBxmHGtKikFEjgWeBa4D6gded2HQ6x6sq+sqXHVT/uvVwVVB/R5GXEWV9jlvD7x+UtD+RwfdXwkcV473NBFkicCUVx1gC7BdRFoC/1sJ7/kecKKI/FVEagA3Aw2iFOME4BYRaSwi9YG7S9tZVdfi6r5fAhap6q+BTbWAmkAusE9EegJnlSGGoSJST9x1FjcGbTsUd7DPxeXEgbgSQb61QHJ+o20IrwPXiEg7EamFS1RfqmqJJaxSlPY5rwkslwca4AcTlICAF4C7RKSDOM1EJDj5mUpgicCU1+3AlbjG2+dwZ8RRFTjY9gGeAjbgziR/xF33EOkYn8XV5f8MzMKd1R/MeFzj7/igmDcDtwLv4BpcL8YltHDcjyuZLAc+BF4Net25wCjg+8A+LYDvgp77CfArsFZEgqt48p8/FVeF807g+Sm4doPyKPFzDlTfDQKG4tpzjg+OU1VfBx4LPOcP4G1cycRUInH/J2OqHhGpjqviuFhVv/Q6HmOqKisRmCpFRLqLyGGB6ox7gb24s2JjTDlZIjBVzanAUlw1Q3fgAlUtqWrIGBMGqxoyxhifsxKBMcb4XJUbdO7II4/UtLQ0r8MwxpgqZfbs2etVNWR36yqXCNLS0sjKyvI6DGOMqVJEpMSr461qyBhjfM4SgTHG+JwlAmOM8bkq10YQSl5eHjk5OezatcvrUKqsxMREkpOTSUgoaWgaY0y8iotEkJOTQ506dUhLS8MNSGnKQlXZsGEDOTk5NG3a9OBPMMbElahVDYnIWBFZJyLzStgugSntlojIXBE5sbzvtWvXLurXr29JoJxEhPr161uJysSMzExIS4Nq1dxtZubBnmFxVEQ02wheppR5XoEeQLPAMhg32mO5WRKoGPv8vOWXA064MQweDCtWgKq7HTy48mPxUxxRqxpS1RkSmJC8BL2BVwPD1H4bGHO9kaqujlZMxsSi/B/6jh3ucf4PHaB/eQeGrsQ4VN1z9uyBvDx3u2cPHHEE1KsHO3fC3LkF6/P3ad8eUlNh3TqYMqVg/fDhBTHk27EDbrgBFi0qWNe/PzRvDgsXwvjxFHP11S6Z/fQT/Pe/xbdfey0ccwzMmgXvvlt8+0svlR7HnXdCnTrw6afwxRfFn3/PPVCzJnz4IcycWXibCDzwgLs/eTLMnl14e61aMGyYu3/LLaHjGDYsgt+PaE6IjJv0el4J294DTg16/CmByb5D7DsYyAKyUlJStKj58+cXW2fKzj7HyrV/v7tNTVV1h9PCyyGHFOx7wQWqHTq45cQT3XLzzQXbzz1XtWNHt6Snu2Xo0ILt3bqpdupUeHn44YLtXbqo1qwZOo4mTVTT0lQbN1Zt0EC1Xj0X20MPueeuXBn6eU895bYvWBB6e0aG2z5rVujtoRaRguW999zzJ08uvD5/+eILt33cuNDbZwdmSX722dDbRUqPY/Vq9/z77gv93G3b3Pbbbiu+rVq1gs9+0KDi2+vWLdheWgxlAWRpCcdqLxuLQ9VFhBwBT1UzcBNik56eXuFR8jIzXTbNzoaUFBg5smKZdfPmzYwfP57rr7++TM8777zzGD9+PPXq1Sv/m5sqITfXnZn+/HPBIgLff+++h6Fs315w/+ijYd8+dwjId8QRBfcbNoQaNQpvr1u34P7hh0NiYuHXT0oqvO+ePaHjyMmByy93Z7cJCe62Zk3o1Mltr1cPHn+8YFv+7Uknue1NmsAHHxR+bkKCKw0AtG3rPoP89SecACtDTF+fmgrLlxdf36sX7N8fOnZwv+3Sft/XXuuWotLSXKnoYHE88EDB2X0oTz7plpJkZLilJKmpoeNIOdjkqWVRUoaIxELpJYLngH5BjxcBjQ72mh07diyW6cpyJjtunGpSUuHMmpTk1pfXsmXLtHXr1sXW7927t/wv6gErEVTc1q2q336r+vzzqnfeWXDWf8UVBd+3hg1VzzrLbVctuUSQmlq5scdKHNH4jVocpZcIvEwE5+Om3xOgC/B9OK8ZTiI444ziy+jRbluTJqG/7PXru+25ucWfezB9+vTRxMREbd++vaanp2u3bt20X79+2rJlS1VV7d27t5544onaqlUrfe655w48LzU1VXNzc3XZsmXaokULHThwoLZq1UrPOecc3bFjR4nvl5GRoenp6dquXTu96KKLdPv27aqqeuWVV+pbb711YL9DguoWHnvsMW3Tpo22a9dO77777pCva4kgfHl5qvPnq+7a5R6/+qrqsccWrk5ISlJdu9Ztz8pSnTat4HGweDvgRCqW1FT3eaamehNDvMXhSSLATY69GsgDcoBrgGuBawPbBRgN/IabFzZk+0DRpaKJoLR6P9XyJYLgEsH06dM1KSlJly5demD7hg0bVFV1x44d2rp1a12/fr2qFk4E1atX1x9//FFVVS+55BJ97bXXSny//Oerqg4bNkxHjRqlqiUngg8++EC7du16IGHkx1OUXxNBOD+y5ctVH3/cndmfcIJqrVruO/Ptt277+++rXnKJ6ogRqm+/rfrrr6r79kU2hsoQK3GYyCstEUSz11C/g2xX4IZovPfnn5e8LSWl5Ho/gCOPLP354ejUqVOhC7NGjRrFO++8A8DKlSv59ddfqV+/fqHnNG3alBNOOAGAjh07sjxUZWjAvHnzuOeee9i8eTPbtm3j3HPPLTWeadOmcdVVV5EUqBQ+Irhy2edC9ZS5+mp45RXYvRuGDIEePWDZMrjrLtfLpG1bOPtsaNMGjj3WPe+889xSXgerx64ssRKHqVxxcWVxWYwcWfiHD67RbOTIyL3HIYcccuD+559/zrRp05g5cyZJSUl069Yt5IVbtWrVOnC/evXq7Ny5s8TXHzBgAJMmTaJ9+/a8/PLLfB7IXDVq1GB/oNVMVdkTaP1TVbtOoATDhhXvmrdnD0ybBl26uAZagK5dYcOGwg20xsQL3w0617+/a6FPTXW9NlJT3eOKnAXVqVOHrVu3hty2ZcsWDj/8cJKSkli4cCHffvtt+d8oYOvWrTRq1Ii8vDwyg64qSUtLY3agQ/LkyZPJy8sD4C9/+Qtjx45lR+CIt3HjxgrHEC9K6rED8M030LOnu1+rliUBE798VyKAyBd/69evzymnnEKbNm2oXbs2Rx111IFt3bt3Z8yYMbRr147mzZvTpUuXCr/fgw8+SOfOnUlNTaVt27YHktCgQYPo3bs3nTp14qyzzjpQMunevTtz5swhPT2dmjVrct555/Hwww9XOI6qLiOjcHfLYBHtmmdMjKtyk9enp6dr0RnKFixYQMuWLT2KKH745XNUdVevjhgB7drBr7+6q1/zJSVVvJRoTKwRkdmqmh5qm++qhoy/7d0Lgwa5JHD11ZCVBc8/H9mqQmOqGl9WDVUVN9xwA19//XWhdTfffDNXXXWVRxFVfRMmwIsvwr33uqtBRaynjDGWCGLY6NGjvQ4hbqi6g36/ftC4MZxxhtcRGRM7rGrIxL3ffoOTT3YjRopYEjCmKCsRmLiWlQXnn+/aBjZv9joaY2KTlQhM3Jo6Fbp1g9q13TUBnTt7HZExsckSgYlLn30Gf/0rNGvmJgVp3tzriIyJXf5MBB7Px3fooYdW6vv5UdeucOutbuaoRo28jsaY2Oa/RBArE5GaiNu3Dx5+2LUF1K7tJksJnpzFGBNa/DUW33ILzJlT8vZvv3XDSgbbsQOuucZdWRTKCSfAM8+U+JJ33303qampB2YoGz58OCLCjBkz2LRpE3l5eTz00EP07t37oOFv27aN3r17F3ve8uXL6dmzJ/PmzQPgiSeeYNu2bQwfPpwlS5Zw7bXXkpubS/Xq1Xnrrbc47rjjDvpe8WTnTrjsMpg0CRo0cBeNGWPCE3+J4GCKJoGDrQ9D3759ueWWWw4kggkTJjB16lRuvfVW6taty/r16+nSpQu9evU66CigiYmJvPPOO8WeV5r+/fszZMgQLrzwQnbt2nVgBFK/2LjRtQfMnAmjRlkSMKas4i8RlHLmDpQ+EWk5JyLo0KED69atY9WqVeTm5nL44YfTqFEjbr31VmbMmEG1atX4/fffWbt2LUcffXSpr6WqDB06tNjzSrJ161Z+//13LrzwQsAlEj/JzoZzz4WlS+HNN+GSS7yOyJiqJ/4SwcFEaUKCiy++mIkTJ7JmzRr69u1LZmYmubm5zJ49m4SEBNLS0kLOQ1BUSc8LnmsAOPBaVW3QwEirVs1Nev7xx3ahmDHl5b/G4mhMSICrHnrjjTeYOHEiF198MVu2bKFhw4YkJCQwffp0VoQqhYRQ0vOOOuoo1q1bx4YNG9i9ezfvvfceAHXr1iU5OZlJkyYBsHv37gPzDsSzOXNc43ByMvz4oyUBYyrCf4kA3EF/+XLYv9/dRmDEsdatW7N161YaN25Mo0aN6N+/P1lZWaSnp5OZmUmLFi3CDC308xISErjvvvvo3LkzPXv2LPR6r732GqNGjaJdu3acfPLJrFmzpsJ/Tyx74w3o1Akee8w9rubPb7ExEWPzEZgDqsLn+OSTcMcdcPrpMHky1KvndUTGVA02H4Gp8vbvdxeI3XGHaxD+6CNLAsZEiv8ai2PEzz//zBVXXFFoXa1atfjuu+88iii2LVoEzz0HN90ETz9t1UHGRFLcJAJVPWgf/VjStm1b5pR24Vsli9Uqwj17XK+gli1h7lw47jjXxm+MiZy4OK9KTExkw4YNMXswi3WqyoYNG2LuGoTff4f0dHjhBff4+OMtCRgTDXFRIkhOTiYnJ4fc3FyvQ6myEhMTSU5O9jqMA375BXr0cOMGpaV5HY0x8S0uEkFCQgJNmzb1OgwTIV9+Cb16QWIizJjhhnoyxkRPXFQNmfixciX85S9w1FFu7CBLAsZEnyUCE1OaNIExY+Drr61KyJjKYonAeCZ4fqB69eC++9z6K6+E+vU9Dc0YX7FEYDxRdH6gLVvgkUdsfiBjvGCJwHhi2LDCA8AC7N3r1htjKpclAuOJ7OyyrTfGRI8lAuOJJk1Cr09Jqdw4jDFRTgQi0l1EFonIEhEZEmJ7qoh8KiJzReRzEYmdK5pMVKjCtm1ukvmkpMLbIjA/kDGmHKKWCESkOjAa6AG0AvqJSKsiuz0BvKqq7YARwCPRisfEhkcegY4d3fSSUZgfyBhTDtEsEXQClqjqUlXdA7wB9C6yTyvg08D96SG2mzgyZoxrDO7UCY44IirzAxljyiGaiaAxsDLocU5gXbCfgL8F7l8I1BGRYj3IRWSwiGSJSJaNJ1Q1vfkmXH899OwJY8faMNLGxJJo/hxDjRNZdHjQO4AzRORH4Azgd2BvsSepZqhquqqmN2jQIPKRmqj67DO44go49VSYMAESEryOyBgTLJqDzuUAwX1DkoFVwTuo6irgIgARORT4m6puiWJMxgNt2sBll8G//gW1a3sdjTGmqGiWCGYBzUSkqYjUBPoCU4J3EJEjRSQ/hn8AY6MYj6lky5ZBXh40bAgvvwyHHeZ1RMaYUKKWCFR1L3Aj8BGwAJigqr+IyAgR6RXYrRuwSEQWA0cB1nkwTvz2G5x8smsXMMbENqlqs3qlp6drVlaW12GYUqxe7doDNm92cwu0Ktpp2BhT6URktqqmh9oWFxPTmNixaZObT2DtWtdIbEnAmNhnicBE1GWXweLF8P777noBY0zss0RgIuqhh9wsY2ef7XUkxphw2WU9psL27XMlAHDDR1xwgbfxGGPKxhKBqRBV+Pvf3RXDX37pdTTGmPKwRGAq5L774Nln4a674LTTvI7GGFMelghMuT3zjGsTGDgQHn3U62iMMeVlicCUy+LFcPvtcPHFblRRCTWylDGmSrBeQ6Zc/vQn+PBDOOMMqF7d62iMMRVhicCUyeefu0nmzz7bXThmjKn6LBGYsM2eDb16wfHHQ1aWzSlgTLywn7IJy8KF0L27m1ns3XctCRgTT+znbA5q5UpXDVStGnzyCTQuOs+cMaZKs6ohc1DPPw9btrj2gWbNvI7GGBNpViIwBzV8uGsf6NDB60iMMdFgicCEtGsXXH01LF3qqoSOP97riIwx0WKJwBSzdy/06wcvvQSzZnkdjTEm2iwRmEJUYfBgmDTJTTbfp4/XERljos0SgTlA1Q0e99JLcP/9cNNNXkdkjKkMlgjMAbt2wYwZcOONLhEYY/zBuo8awJUGateG6dMhMdEGkTPGT6xEYJgwAXr0gG3bICnJrho2xm/sJ+9zH30El18O27dbAjDGr+yn70OZmZCW5g78PXrA0Ue78YOSkryOzBjjBUsEPpOZ6bqHrljh2gVUYf36gsnnjTH+Y4nAZ4YNgx07Cq/budOtN8b4kyUCn8nOLtt6Y0z8s0TgI6oltwOkpFRuLMaY2GGJwEeeecb1DkpIKLw+KQlGjvQmJmOM9ywR+MRHH8Edd8BFF8HYsZCa6i4aS02FjAzo39/rCI0xXrEri31g8WLo2xfatIFXXoFDD3XXDhhjDFiJwBcWL4a6dWHyZJcEjDEmmCUCH+jZ0yWDtDSvIzHGxKKoJgIR6S4ii0RkiYgMCbE9RUSmi8iPIjJXRM6LZjx+c889rj0AoFYtb2MxxsSuqCUCEakOjAZ6AK2AfiLSqshu9wATVLUD0Bf4T7Ti8Ztx41xPoKwsryMxxsS6aJYIOgFLVHWpqu4B3gB6F9lHgbqB+4cBq6IYj298/z0MHAjdurlZxowxpjTRTASNgZVBj3MC64INBy4XkRzgA+DvoV5IRAaLSJaIZOXm5kYj1rixahVccAE0agRvvVX8mgFjjCkqmokg1NQmWuRxP+BlVU0GzgNeE5FiMalqhqqmq2p6gwYNohBq/Hj/fdi6FaZMgSOP9DoaY0xVEM1EkAM0CXqcTPGqn2uACQCqOhNIBOzwVQGDBrkeQm3beh2JMaaqiGYimAU0E5GmIlIT1xg8pcg+2cBZACLSEpcIrO6nHMaMgW++cfcbNfI2FmNM1RK1RKCqe4EbgY+ABbjeQb+IyAgR6RXY7XZgkIj8BLwODFDVotVH5iA++ACuvx7+Y32ujDHlIFXtuJuenq5Z1ifygIULoXNnOPZY+OorOOQQryMyxsQiEZmtqumhttmVxVXYpk3Qu7e7WGzyZEsCxpjysUHnqrDRo2HZMvjsM5tPwBhTflYiqMKGDoWvv4ZTT/U6EmNMVRZWIhCRLiJSJ+hxHRHpHL2wTGmmTIGVK6FaNTjpJK+jMcZUdeGWCJ4FtgU93h5YZyrZzJlwySVw991eR2KMiRfhJgIJ7tapqvux9oVKl5MDF14ITZrAv//tdTTGmHgRbiJYKiI3iUhCYLkZWBrNwExhO3a4MYR27HBVQ0cc4XVExph4EW4iuBY4GfgdN3REZ2BwtIIyxY0YAT/8AOPHQ6uig3kbY0wFhFW9o6rrcENEGI8MHeoahnv29DoSY0y8CSsRiMhLFB85FFW9OuIRmUK+/95NOl+3Lvztb15HY4yJR+FWDb0HvB9YPsVNJrOt1GeYCvvlFzjrLPh7yFkajDEmMsKtGvpv8GMReR2YFpWIDAAbNkCvXnDoofDAA15HY4yJZ+XtAtoMsEENoiQvDy691HUX/eILSE72OiJjTDwLt41gKwVtBAqsBe6KVlB+d++9bvygl1+GLl28jsYYE+/CrRqqIyJH4EoCifmroxaVzw0aBA0bwpVXeh2JMcYPwi0RDARuxk03OQfoAswEzoxeaP6Tne2uGj7uOLjtNq+jMcb4Rbi9hm4GTgJWqOqfgQ7YlJIRlZ0N6ekwZIjXkRhj/CbcRLBLVXcBiEgtVV0INI9eWP6yfbubYGb3brjqKq+jMcb4Tbi9hnJEpB4wCfhERDYBq6IXln+ouoP/Tz/B++9DixZeR2SM8ZtwG4svDNwdLiLTgcOAqVGLykdGjoS33oJ//hN69PA6GmOMH5X5OgJV/SIagfhV167uyuHbb/c6EmOMX9mcAh7ZsQOSktwQEmed5XU0xhg/szmLK1FmJqSluSkmDzsMBgzwOiJjjLFEUGkyM2HwYFixwjUQ790Lb77p1htjjJcsEVSSYcNcdVCwXbvcemOM8ZIlgkqSnV229cYYU1ksEVSSlBLGai1pvTHGVBZLBJVk5EjXSyhYUpJbb4wxXrJEUEn+9jfIyIDUVBBxtxkZ0L+/15EZY/zOriOoJDffDPPnw9KlrvuoMcbECjskVYI//nDdRJs1syRgjIk9dliqBOPGuRFGr7vO60iMMaY4SwRRpgpjxsCJJ7r5BowxJtZENRGISHcRWSQiS0Sk2JQrIvK0iMwJLItFZHM04/HCzJnw889w7bWukdgYY2JN1BqLRaQ6MBo4B8gBZonIFFWdn7+Pqt4atP/fcTOfxZUTToCXXoKLL/Y6EmOMCS2aJYJOwBJVXaqqe4A3gN6l7N8PeD2K8XgiKckNLnfooV5HYowxoUUzETQGVgY9zgmsK0ZEUoGmwGclbB8sIlkikpWbW3WmSn7tNXj6adi/3+tIjDGmZNFMBKFqxLWEffsCE1V1X6iNqpqhqumqmt6gQYOIBRhN+/fDgw/Cf/9rXUaNMbEtmoeoHKBJ0ONkSp7nuC9xVi00fTr8+qt1GTXGxL5oJoJZQDMRaSoiNXEH+ylFdxKR5sDhwMwoxlLpxoyB+vXd0BLGGBPLopYIVHUvcCPwEbAAmKCqv4jICBHpFbRrP+ANVS2p2qjKWb0aJk2Cq66CxESvozHGmNJFdawhVf0A+KDIuvuKPB4ezRi8sHEjnH66m5HMGGNinQ06FwWtW8Onn3odhTHGhMf6s0TYokWuasgYY6oKSwQRdttt0KWLXTtgjKk6LBFE0PLl8OGHcOWVdu2AMabqsMNVBD3/vBtYbuBAryMxporLzIS0NHdGlZbmHpuoscbiCNmzB158Ec4/3yakN6ZCMjNdl7sdO9zjFSsKuuDZ3K5RYSWCCPnuO8jNtSuJjamwYcMKkkC+HTvcehMVViKIkNNOg2XLoHHIYfWMMWHLzi7belNhViKIgPxrolNSoHp1b2MxpspauhSuvrrgB1VUrVrw+eclbzflZokgAu68E3r1qkJdRq0hzsSS5cth0CBo3hzGj4fu3aF27cL7JCRAzZrw5z/DqafC++9bQoggSwQVtHMnjB3rvrdVostofkPcihXuh5TfEGfJwFS27Gw3h+uf/gSvvuoa2JYudX2wn38eUlNdN7zUVDfN35o1MHo0/P479OwJHTrAm2/CvpCj15uyUNUqtXTs2FFjycsvq4LqZ595HUmYUlJcwEWX1FSvIzN+sXKl6vXXq9asqZqQ4O6vXBn+8/fsUX3lFdUWLdx3t1kz1RdeUN29O3oxxwEgS0s4rlaFc9iYNmaMK9F26+Z1JAcxbx7ce2/JDW4rVsCoUbBggf+K3FZVVjlWrYKbboLjj4eMDDc875Il7iw/OTn810lIgP/5H/jlFzfzU5067uKd445z3+GiPY7MwZWUIWJ1iaUSwY8/uhOSp5/2OpISLFyo+sADqq1auUCrVVNNTAxdIqhRo+B+48aqAwaoZmaqrghX7LAAABGMSURBVF3r9V8RXePGqSYlFf4skpLcehMZq1er3nKL++5Vr646cKDqsmWRe/39+1WnTlU9/XT3/zvySNWRI1U3bYrce8QBSikReH5gL+sSS4lg/XrVRx9V3bDB60iC/Pab6sMPq7Zv7/69Iu4HMnq06po1pR/4li1TzchQvfhi1cMPL9jevr3qHXeofvSR6o4dXv+FkbF7t+qcOar164dOjEcdpZqdrbp3r9eRVl1r16refrtq7douAVx1lft+RtOXX6r26OH+h3Xrqv7jH/F/MhMmSwTxLjtb9YknVE86qeBA1rWr6jPPqObkFN9/3DjXJiDibkOd/e7dq/r99+7Mqls3V5cLqrVqqZ59tupjj6n+8IPqvn3R/usqLjdXddo01SefVL3iCtV27Qr+noMtCQmqxx3n/uaBA93nkZmp+s037kx3/36v/7rYk5uretdd7gSjWjXV//kf1V9/rdwYfvhB9ZJL3He8dm3Vm25yv5OqKpzf7EFYIoiCKVPc/8Kz4+CqVar/+pfqyScXHLQ6dlR9/HHV5csj/37btql+8IHqrbeqtmlT8J4NGqj27av64ove/9D27lWdP1/19ddVhwxxZ4bHHFP4wN6okWr37m7766+7x6ESQMOGqmPGuP369FHt3NmtK7pfYqJrtOzeXfW661yCnDBBddYsd0AMN1FE4IfuufXr3Rn4IYe4v6N/f1c96aWFC11JpEYNl9Svvlp10SJvYyqrCFVflpYIxG2vOtLT0zUrK8vTGFRdzzUR+OEHd1sp1q1zjWNvvgkzZrhA2rWDPn3g0ktdI1xlWbUKpk2DTz5xt2vWuPUtWsA557ilWzfXkBcNW7bA3Lnw008Fy7x5rj8vQI0a0KoVtG9feGnQoPDrFB3XBiApyTVmhhrXZvt217C+bJnr/170duPGwvsfeqhrgG7atOA2+P5hh5U9hlizaRM89RT861+wbZv7Pt53H7Rs6XVkBbKz4YknXLfU3bvhkktg6FD3nYg1W7a47/LPP7vlxRddzEWlprrvXJhEZLaqpofcWFKGiNUlFkoEM2e6pDxmTCW82YYNrmvcOee4elZwZ6D33+/OfmPB/v2qc+e6qpfu3V1RPL8B+tRTXYP1N9+o5uUVfl44Z8H79rl65bffdn9z796qaWmFz47q11c980xXWnn5ZVf3X5auhJE8G9+82b3/pEmuF8HNN6v26uWqo+rUKV6iqFev5GqqlJTyx1EZNm1y/5O6dV28l1yiOm+e11GVbs0aV8rL/1+cf77q1197E8uePao//6w6frwrSZ1/fvHu3fmfbahFpExvh5UIImvAAHdivmpVlE54t2yByZPdmf/HH8Peva5rXJ8+bmnbthKLIeWwezd8/bUrLXzyiSs2qbqz3zPPdKWFnTtdd9bgs+DatWHIEDj66IKz/LlzYetWt71aNXfxUdGz/GOOie3PI5+qO3tetqxwSeI//yn5OR06uFJW8+butkULaNbMlRi88scf7uz/qadg82a46CK4/35XOq0qNm923Vaffho2bIAzznAlhHPOifx3SRVWriw4w89fFi6EvDy3T40a7n/btm3hJSXFlRxXrCj+uhEsEVgiKKONG93AcgMGwLPPRvCFt22Dd991B/+pU93BNCWl4OB/4olV42AXyoYNbhLn/MQQ6ktdVN267sASfMBv08bbA2C0pKWF/kzq1IFTTnHzny5fXvj6jtTUwskh/36jRtH7nmzd6vrpP/mkS2i9e8Pw4XDCCdF5v8qwfburLnriCXfFcnq6Swi9e5dvqIDNmwtX6+QvW7YU7JOSUvyA37y5G0IjlAhVHVrVUATNmaPaurW7LbOiVRBjx6pOnOiK1PnVKccc46oTvvkmPnuk7N+vunhx6cXdpUvj828vSTiNgTt2qP70k2uIHjFC9bLLXOeAQw4p/Lw6dVTT01Uvv1z1wQdV33rLVdvt3Fn++LZuVX3kkYKutj17qmZlVfzvjiW7dqk+/7zrIQaqLVuqvvqqu4I5VLXh7t3u/zFunOrdd6ued55qkyaF/xeHHeaqRq+7TvU//1H96itXdVgeUe41ZCWCclAtx0lXqKyer0ED13jVp48bUKtKDFpUQSWdBZexuBs3MjPdePvZ2e6MceTI8M72VF0d5cKFblm0qOA2+CpyEVfFkF9yCC5NNGxY8IUOjiM52ZVIpk2D9euhRw9XAujUKSofQUzYuxcmToSHH3Zn8iKFS2LVq7tS15o1bl9wVzq3bFn8LD85OaZK8VY1FCE5OVCvnusIUmapqaGHd2jY0BVJa/hsaoiq3lOmKti+HRYvLpwc8m/ze1eBa7tp0cId0L77rqDeOl+bNq76pEuXyo3fS6pw1FFutqmiateGW25xB/t27Vy7VUJC5cdYRpYIIqRPH/j+e/jttzKctKvClClwwQWht4tUofGrI6y8Z8GmYvbvd2c1RUsRX3wReiRPv5bSqlULPe5WFf3NWiKIgLVrXUnvpptcW1lYvvjC9YL59lt3xp9flAzm1x+ZiT1xduCrsDirviwtEfigMjoyxo51x/H8ObRLNWeOq0/t1s2d7WZkuBco2uMlKcmdBRsTC1JSyrY+3o0c6ZvfrCWCMOzbB88957rAN29eyo5LlsBll7m+3999B48/7tYNGgRXXOESQvBkG1YfbmKJjw58Yenf3ze/WasaCsOMGe56kwkTXOeeYlavhgcfdA1qCQlw661u/sp69So1TmMqzNpt4pa1EUTAjz+6zhOFOgds3gz//Cc88wzs2ePO/O+913UvM8aYGFJaIvBZn8Xy69Ah6MHOnfDvf8Mjj7grLPv1gxEjKnfQN2OMiRBrIziIkSPdjHr79+Nai59/3o31ctdd0LmzG0dn/HhLAsaYKssSQSny8tyJf+46pdrbE6F1a9dtqEkTmD4dPvywSFHBGGOqnqgmAhHpLiKLRGSJiAwpYZ9LRWS+iPwiIuOjGU9ZTZkCrddMY9ziTq6VuEYNmDQJvvmmCsxWb4wx4YlaG4GIVAdGA+cAOcAsEZmiqvOD9mkG/AM4RVU3iUjDaMVTZrNmkTZ4KNOYhu5JgZdfhssvd2ONGGNMHIlmiaATsERVl6rqHuANoHeRfQYBo1V1E4CqrotiPOFZtMid/XfqRMrGOXzc42lk0SK48kpLAsaYuBTNRNAYWBn0OCewLtifgD+JyNci8q2IdA/1QiIyWESyRCQrN9QgUJGQk+O6f7ZuDVOnsvW2+3n6ht9o++ItkJgYnfc0xpgYEM3uo6HGXy160UINoBnQDUgGvhSRNqq6udCTVDOADHDXEUQ0yo0b4dFH4f/+z11CfOONMHQodRo25OGIvpExxsSmaJYIcoAmQY+TgVUh9pmsqnmqugxYhEsM0bd9uxtz/Nhj3exEl17qhux95hm+WNCQTz4JPf6WMcbEm2gmgllAMxFpKiI1gb7AlCL7TAL+DCAiR+KqipZGPJLMTDeSYLVqbryQAQPcHMDDhrmxI+bOhVdecfvgVt9wgyUCY4w/RK1qSFX3isiNwEdAdWCsqv4iIiNwU6ZNCWz7i4jMB/YBd6rqhogGUnQClOxsd9Bv3hzefhtOPrnQ7j//7OZdf+IJf0wUZowx8T/WUEljiqekhFx/ww3w4otu0rD69csfpzHGxBJ/z0cQanpIgJUri63atg1ee801F1gSMMb4RfwngjJMtrFokZuP+NproxyTMcbEkPhPBGWYbKNjR1db1LVrJcVmjDExIP4TQZizDG3a5AYXTUhwuxljjF/4Yz6C/v0POsvSHXe4seTmzbORJIwx/hL/JYIwbN4Mr78Op59uScAY4z+WCIBXX3WTjlkjsTHGj3yfCFRhzBjo1MnmmDHG+JM/2ghK8c03sGABjB3rdSTGGOMN3yeCrl1h6lQ47TSvIzHGGG/4PhFUqwbnnut1FMYY4x1ftxE8+yzceaebhsAYY/zKtyWC/fvhn/90I01Yl1FjjJ/5tkTwySewbBlcd53XkRhjjLd8mwjGjIEGDeDCC72OxBhjvOXLRJCTA+++C9dcAzVreh2NMcZ4y5eJIC8P+vaFQYO8jsQYY7zny8bipk1h3DivozDGmNjguxLBjz+6ueqNMcY4visRDBkC8+fD8uXWbdQYY8BnJYLffoOPP3ZtA5YEjDHG8UUiyMyEtDQ4/nj3+IgjPA3HGGNiStxXDWVmwuDBsGNHwbq774bDDz/opGXGGOMLcV8iGDascBIA93jYMG/iMcaYWBP3iSA7u2zrjTHGb+I+EaSklG29Mcb4TdwngpEjISmp8LqkJLfeGGOMDxJB//6QkQGpqSDibjMyrKHYGGPyxX2vIXAHfTvwG2NMaHFfIjDGGFM6SwTGGONzlgiMMcbnLBEYY4zPWSIwxhifE1X1OoYyEZFcYIXXcVTQkcB6r4OIIfZ5FLDPojD7PAqryOeRqqoNQm2ocokgHohIlqqmex1HrLDPo4B9FoXZ51FYtD4Pqxoyxhifs0RgjDE+Z4nAGxleBxBj7PMoYJ9FYfZ5FBaVz8PaCIwxxuesRGCMMT5nicAYY3zOEkElEpEmIjJdRBaIyC8icrPXMXlNRKqLyI8i8p7XsXhNROqJyEQRWRj4jnT1OiYvicitgd/JPBF5XUQSvY6psojIWBFZJyLzgtYdISKfiMivgdvDI/V+lggq117gdlVtCXQBbhCRVh7H5LWbgQVeBxEj/gVMVdUWQHt8/LmISGPgJiBdVdsA1YG+3kZVqV4GuhdZNwT4VFWbAZ8GHkeEJYJKpKqrVfWHwP2tuB96Y2+j8o6IJAPnAy94HYvXRKQucDrwIoCq7lHVzd5G5bkaQG0RqQEkAas8jqfSqOoMYGOR1b2BVwL3XwEuiNT7WSLwiIikAR2A77yNxFPPAHcB+70OJAYcC+QCLwWqyl4QkUO8Dsorqvo78ASQDawGtqjqx95G5bmjVHU1uJNKoGGkXtgSgQdE5FDgv8AtqvqH1/F4QUR6AutUdbbXscSIGsCJwLOq2gHYTgSL/lVNoP67N9AUOAY4REQu9zaq+GWJoJKJSAIuCWSq6ttex+OhU4BeIrIceAM4U0TGeRuSp3KAHFXNLyFOxCUGvzobWKaquaqaB7wNnOxxTF5bKyKNAAK36yL1wpYIKpGICK4OeIGqPuV1PF5S1X+oarKqpuEaAT9TVd+e8anqGmCliDQPrDoLmO9hSF7LBrqISFLgd3MWPm48D5gCXBm4fyUwOVIv7IvJ62PIKcAVwM8iMiewbqiqfuBhTCZ2/B3IFJGawFLgKo/j8YyqficiE4EfcL3tfsRHw02IyOtAN+BIEckB7gceBSaIyDW4RHlJxN7Phpgwxhh/s6ohY4zxOUsExhjjc5YIjDHG5ywRGGOMz1kiMMYYn7NEYEyAiOwTkTlBS8Su7BWRtOCRJI2JJXYdgTEFdqrqCV4HYUxlsxKBMQchIstF5DER+T6wHB9Ynyoin4rI3MBtSmD9USLyjoj8FFjyh0aoLiLPB8bY/1hEagf2v0lE5gde5w2P/kzjY5YIjClQu0jVUJ+gbX+oaifg37hRUwncf1VV2wGZwKjA+lHAF6raHjde0C+B9c2A0araGtgM/C2wfgjQIfA610brjzOmJHZlsTEBIrJNVQ8NsX45cKaqLg0MGrhGVeuLyHqgkarmBdavVtUjRSQXSFbV3UGvkQZ8EphUBBG5G0hQ1YdEZCqwDZgETFLVbVH+U40pxEoExoRHS7hf0j6h7A66v4+CNrrzgdFAR2B2YCIWYyqNJQJjwtMn6HZm4P43FEyf2B/4KnD/U+A6ODAnc92SXlREqgFNVHU6bpKeekCxUokx0WRnHsYUqB00Kiy4+YPzu5DWEpHvcCdP/QLrbgLGisiduNnF8kcLvRnICIwSuQ+XFFaX8J7VgXEichggwNM2RaWpbNZGYMxBBNoI0lV1vdexGBMNVjVkjDE+ZyUCY4zxOSsRGGOMz1kiMMYYn7NEYIwxPmeJwBhjfM4SgTHG+Nz/A+36b9mwzudNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_metric(dfhistory, metric):\n",
    "    train_metrics = dfhistory[metric]\n",
    "    val_metrics = dfhistory['val_'+metric]\n",
    "    epochs = range(1, len(train_metrics) + 1)\n",
    "    plt.plot(epochs, train_metrics, 'bo--')\n",
    "    plt.plot(epochs, val_metrics, 'ro-')\n",
    "    plt.title('Training and validation '+ metric)\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(metric)\n",
    "    plt.legend([\"train_\"+metric, 'val_'+metric])\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# 观察损失和准确率的变化\n",
    "plot_metric(dfhistory,\"loss\")\n",
    "plot_metric(dfhistory,\"auc\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "出现过拟合了，其实，并且比较严重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:29:06.517773Z",
     "start_time": "2020-11-30T03:29:06.504809Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测\n",
    "y_pred_probs = model(torch.tensor(test_x).float())\n",
    "y_pred = torch.where(y_pred_probs>0.5, torch.ones_like(y_pred_probs), torch.zeros_like(y_pred_probs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-30T03:29:06.548690Z",
     "start_time": "2020-11-30T03:29:06.520763Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.data\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
